
% comparison of hyps learners vs exact:
% check how well each does over a range of hyperparameters
% uses cvwl

addpath('gpml-matlab-v3.1-2010-09-27');
startup;
% warning('off','all'); % gotta love playing with fire
n_reps = 3;
n = 1000;
ms = [20,50,100];
mn = length(ms);

output = 'expt6_results';

results = zeros(mn, 7); % frac right, error; then n
data = zeros(n_reps,3); % how well method 1 does each time; how well method 2 does
for expt = 1:mn
  fprintf('\nExpt: %i\n', expt);
  for trial = 1:n_reps
    m = ms(expt);
    fprintf('Trial: %i\n',trial);
    gendata(normrnd([-0.3; -0.6 - trial / 3.0; -0.4], 0.2), n);

    disp('CV with learning and approximations')
    CVP(m);
    tmp = load('cv_results');
    data(trial, 1) = tmp(1);
    data(trial, 2) = tmp(2);
    
    disp('CV')
    cross_validate();
    tmp = load('cv_results');
    data(trial, 3) = tmp(1);
    
    D = load('gp_data');
    dlmwrite(strcat('gp_data_',int2str(m)), D);
    D = load('points');
    dlmwrite(strcat('points_',int2str(m)), D);

  end
  av = sum(data(:,1)) / n_reps;
  results(expt,1) = av;
  vars = (data(:,1) - av) .^ 2;
  results(expt,2) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  av = sum(data(:,2)) / n_reps;
  results(expt,3) = av;
  vars = (data(:,2) - av) .^ 2;
  results(expt,4) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
  av = sum(data(:,3)) / n_reps;
  results(expt,5) = av;
  vars = (data(:,3) - av) .^ 2;
  results(expt,6) = sqrt(sum(vars)/n_reps) / sqrt(n_reps);
  
%  tmp = load('learnt_hyps_error');
%  results(expt, 10) = tmp;
  
  %dlmwrite(output, results);
end
%error('because I say so')
results(:, 7) = n * ones(mn, 1); % just as an FYI
dlmwrite(output, results);
% note that it's in row form: method 1 fraction correct, the variance thereof, time; sim. for method 2, 3; n
disp('Done');